#!/usr/bin/env python
"""
A basic test-only script that clones the output provided arguments and
options similar to hexdump(1).

 Copyright (c) 2010 hexdump Project. All rights reserved
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:
 1. Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
 2. Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.
 
 THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 SUCH DAMAGE.

This relatively simple script currently handles -e provided arguments.

TODO list (priority-order):

1. Implement shared buffering so that multiple format strings can share the
   same buffer, like with the -C option. That way multiple format units can do
   work on the same data in serial.
2. The script should be expanded to -[bcCdox] as they're all specialized cases
   of -e.
3. Implement wide-character support properly, like NetBSD's hexdump(1) does.
4. od(1) style option handling should be implemented; all the hexsyntax related
   pieces should be split off of this file into proper modules.
"""

import optparse
import re
import struct
import sys

# XXX (gcooper): make INT_SIZE variable according to the value returned from
# platform.architecture() to conform to NetBSD's sizeof(int) paradigm.
#
# >>> import platform
# >>> platform.architecture()
# ('64bit', 'ELF')
# >>>
#
# The code will need to be simultaneously updated in the C code to match this
# behavior.
CHAR_SIZE, INT_SIZE = struct.calcsize('c'), struct.calcsize('i')
BITS_IN_A_BYTE = 8

# Please see Format.__init__ for the full gory details about the possible
# format classes discussed here.
#
# XXX (gcooper): these really should be in a doc string for the initializer
#                or something similar so that it's viewable via pydoc.
FORMAT_NONE, FORMAT_NORMAL, FORMAT_OFFSET_ITER, FORMAT_OFFSET_END = range(4)
FORMAT_DEFAULT_CHARSET, FORMAT_NONPRINTING_CHARS = range(FORMAT_OFFSET_END+1, 6)
FORMAT_ASCII_PRINT = FORMAT_NONPRINTING_CHARS + 1

# Character/string-style format expression // offset regular expression.
EXPR_RE = re.compile('(?P<leading_text>[^%]*)(?P<percent>%+)'
                     '(?P<field_width_and_precision>(\d+\.)?\d+)?'
                     '(?P<qualifier>(_([aA][dox]|[cpu])|[cdiouxXDOUeEfFgGs]))'
                     '(?P<trailing_text>.*)')

EXPR_MULTIPLE_QUALIFIER_RE = re.compile('%+')

# The first column is always 0.
QUALIFIER_TO_VALID_BYTECOUNT = {
    'eEfgG'  : [ 8, 4, 8 ],
    'diouxX' : [ 4, 1, 2, 4 ],
    'c'      : [ 1, 1 ]
}

# For %_u output...
ORD_TO_HUMANIZED = {
    0   : 'nul',
    1   : 'soh',
    2   : 'stx',
    3   : 'etx',
    4   : 'eot',
    5   : 'enq',
    6   : 'ack',
    7   : 'bel',
    8   : 'bs',
    9   : 'ht',
    10  : 'lf',
    11  : 'vt',
    12  : 'ff',
    13  : 'cr',
    14  : 'so',
    15  : 'si',
    16  : 'dle',
    17  : 'dc1',
    18  : 'dc2',
    19  : 'dc3',
    20  : 'dc4',
    21  : 'nak',
    22  : 'syn',
    23  : 'etb',
    24  : 'can',
    25  : 'em',
    26  : 'sub',
    27  : 'esc',
    28  : 'fs',
    29  : 'gs',
    30  : 'rs',
    31  : 'us',
    127 : 'del'
}

# Escaped control sequences and their unescaped counterparts.
#
# See Format.unescape_fmt
#
ESCAPED_CONTROL_SEQUENCES = {
    '\\0' : '\0',
    '\\a' : '\a',
    '\\b' : '\b',
    '\\f' : '\f',
    '\\n' : '\n',
    '\\r' : '\r',
    '\\t' : '\t',
    '\\v' : '\v'
}

REPEAT_ASTERISK = '*\n'

#
# Can't use .tell() because some file descriptors like sys.stdout don't have a
# `sensical' offset from the beginning as they're shared files.
#
# Furthermore, this can span multiple files, which means that we couldn't run
# .tell() on a single file to begin with without keeping an overall offset..
#
# offset = 0
#

if sys.byteorder not in ( 'little', 'big' ):
    # AIIEEEE!!! Middle endian (ARM...)!
    sys.exit('Middle endian is not supported!')

class Format(object):

    """ A format string object """

    def __init__(self, fmt, iteration, byte_count):

        try:
            self.byte_count = int(byte_count)
        except TypeError:
            self.byte_count = 1
        try:
            self.iteration = int(iteration)
        except (TypeError, ValueError):
            self.iteration = 8
        self.fmt = fmt
        self.precision = None
        self.field_width = 0

        percent_groups = EXPR_MULTIPLE_QUALIFIER_RE.findall(fmt)

        if 1 < len(percent_groups):

            # We have a potential format string. Let's make sure that the
            # end-user didn't insert multiple format qualifier tokens in the
            # format string..

            found_a_format_str = False

            for percent_group in percent_groups:

                if (len(percent_group) % 2 != 0):

                    # Format string (!)

                    if found_a_format_str:
                        # Oops -- the end-user specified multiple % qualifiers
                        # in the same format string! This is illegal as per
                        # hexdump(1) usage.
                        raise ValueError('multiple qualifier tokens in '
                                         'the same format string: %s' % fmt)

                    found_a_format_str = True

        # We want to allow the end-user to specify any kind of quantifier 
        format_match = EXPR_RE.match(fmt)

        #sys.stderr.write('``%s`` : ' % (fmt, ))

        #if format_match:
        #    for key, value in format_match.groupdict().items():
        #        sys.stderr.write('``%s`` = ``%s`` ' % (key, str(value), ))

        #sys.stderr.write('\n')
        #sys.stderr.flush()

        if format_match and format_match.group('percent'):

            # Do we have a valid format string...?

            if format_match.group('field_width_and_precision'):
                fwap_g = format_match.group('field_width_and_precision')
                fwap_g = fwap_g.split('.')
                if len(fwap_g) in ( 1, 2 ):
                    try:
                        int(fwap_g[0])
                        self.field_width = fwap_g[0]
                    except:
                        self.field_width = 0
                    if len(fwap_g) == 2:
                        try:
                            # Try to convert the value, but don't set it
                            # because this could remove leading zeroes, etc.
                            int(fwap_g[1])
                            self.precision = fwap_g[1]
                        except:
                            self.precision = 0
                else:
                    raise ValueError('invalid field width / precision '
                                     'specified: %s' % fmt)

            if (format_match.group('qualifier') in
                's' + ''.join(QUALIFIER_TO_VALID_BYTECOUNT.keys())):

                # Standard format string.

                self.format_type = FORMAT_NORMAL

                for qg, qv in QUALIFIER_TO_VALID_BYTECOUNT.items():

                    if 0 <= qg.find(format_match.group('qualifier')):
                        # Let's set the default precision according to the
                        # default byte_count in the corresponding qualifier
                        # group.
                        self.field_width = qv[0]
                        break

                if not self.precision:

                    if format_match.group('qualifier') == 's':

                        assert self.byte_count != 0, \
                            ('Precision and/or byte_count must be specified '
                             'and greater than with %s qualifier)')

                    else:
                        # An empty precision -- need to determine what the
                        # appropriate size is based on the qualifier specified.

                        self.precision = 0

            elif format_match.group('qualifier'):

                # `_' was present, so it's a non-printf(3) style format
                # expression.

                cqg = format_match.group('qualifier')

                custom_text_qualifier = cqg[0] == '_' and cqg[1] in 'cpu'
                standard_format_qualifier = cqg[0] in 'cdiouxXDOUeEfFgGs'

                #print custom_text_qualifier, standard_format_qualifier

                if not self.precision:

                    # Precision not set... let's default to something here.

                    if custom_text_qualifier:
                        self.precision = CHAR_SIZE
                    else:

                        # XXX (gcooper): I know this seems cheap, but all
                        # %_[aA] format qualifiers are 4-bit integer aligned
                        # in FreeBSD's hexdump(1). It's sizeof(int) in NetBSD's
                        # hexdump though. See the comment near the INT_SIZE
                        # definition for more details of what to do next.
                        self.precision = INT_SIZE

                if not standard_format_qualifier:

                    # Common preamble for address based qualifiers and custom
                    # text based qualifiers.
                    self.fmt = format_match.group('percent')

                    if self.field_width is not None:
                        self.fmt += str(self.field_width) + '.'

                    if self.precision is not None:
                        self.fmt += str(self.precision)

                if custom_text_qualifier:

                    if cqg[1] == 'c':

                        # Default charset, with non-printable ala octal or
                        # escaped values.
                        self.format_type = FORMAT_DEFAULT_CHARSET

                    elif cqg[1] == 'p':
                        # Default charset, with non-printable ala `.'.
                        self.format_type = FORMAT_NONPRINTING_CHARS

                    elif cqg[1] == 'u':
                        # US ascii output.
                        self.format_type = FORMAT_ASCII_PRINT

                    self.fmt += 's'

                elif not standard_format_qualifier:

                    # An address-based format string.

                    if cqg[1] == 'A':
                        # A non- iterator-style offset qualifier.
                        self.format_type = FORMAT_OFFSET_END
                    else:
                        # An iterator-style offset qualifier.
                        self.format_type = FORMAT_OFFSET_ITER

                    self.fmt += cqg[2:]

                if format_match.group('leading_text'):
                    self.fmt = format_match.group('leading_text') + self.fmt
                if format_match.group('trailing_text'):
                    self.fmt += format_match.group('trailing_text')

                if (custom_text_qualifier or standard_format_qualifier):
                    if not self.byte_count:
                        self.byte_count = int(self.precision)
                else:
                    # Address-based format strings don't consume bytes.
                    # XXX (gcooper): value should be properly ignored in this
                    # case.
                    self.byte_count = 0

            else:
                # Make sure we don't hit this case...
                assert False, 'Unhandled case for: %s' % (str(fmt), )

            #print 'fmt: ``%s``, byte_count: %d, iteration: %d' %\
            #    (self.fmt, self.byte_count, self.iteration, )

        else:
            # There aren't any precision qualifiers.
            self.byte_count = 0
            self.format_type = FORMAT_NONE

        self.fmt = self.unescape_fmt(self.fmt)

        #print '[3]   fmt: ``%s``, self.fmt: ``%s``' % (fmt, self.fmt, )

    def conv_str(self, fd, offset):
        """ Print out the formatted string, as per the format string
            qualifiers specified with the current Format object.

            This method returns a tuple in the following form:

                number_of_bytes_consumed, output_string
        """

        size = 0

        if self.format_type in ( FORMAT_OFFSET_ITER, FORMAT_OFFSET_END ):

            # Apply the offset string.

            if self.format_type == FORMAT_OFFSET_END or not fd.closed:
                out = self.fmt % (offset, )

        else:

            # We have to consume X number of bytes in the data stream.

            if self.byte_count:

                out_raw = fd.read(self.byte_count * self.iteration)

                size = len(out_raw)

                if size == 0:
                    fd.close()

            if self.format_type == FORMAT_NONE:
                # Print out the string verbatim.
                out = self.fmt
 
            else:

                # Time to do divvy up the strings for the following format
                # unit types: FORMAT_NORMAL, FORMAT_DEFAULT_CHARSET,
                # FORMAT_NONPRINTING_CHARS, and FORMAT_ASCII_PRINT.

                out = ''
                str_offset = 0 
                substring_len = -1 

                while substring_len:

                    # Splice off the minimum number of characters from the
                    # raw output string.
                    substring_len = min(len(out_raw[str_offset:]),
                                        self.byte_count)

                    if substring_len != 0:
                        substring = out_raw[str_offset:str_offset+substring_len]
                        #print 'out_raw: ``%s``, substring: ``%s``' % \
                        #    (out_raw, substring, )
                        out += self.conv_subset(substring)

                        #print '[%d] len(out_raw) = %d' % \
                        #    (str_offset, len(out_raw), )

                        str_offset += substring_len

                # END conv_out iteration.

            # END standard format string.

        return size, out

    def conv_subset(self, subset):
        """ Convert a subset of characters to their proper to-print format.

        The goal is to take the data from subset, and effectively divide and
        conquer it so the problem statement simplifies itself to the smallest
        discrete block possible.
        """

        conv_out_str = ''

        if self.format_type == FORMAT_NORMAL:
            # XXX (gcooper): this is a really brute force means of doing
            # things. We have the means to determine what the qualifier is ahead
            # of time, so we should use that knowledge to do the Right Thing
            # (tm).
            try:
                conv_out_str = self.fmt % (subset, )
            except TypeError:
                int_val = 0
                bytes = map(ord, subset)
                if sys.byteorder == 'little':
                    bytes.reverse()
                for last_byte in bytes:
                    int_val  = (int_val << BITS_IN_A_BYTE) + last_byte
                # Add more data; shift the previous data over.
                conv_out_str = self.fmt % (int_val, )

                #print 'self.fmt, int_val, conv_out_str, ' \
                #      '``%s`` %% %d -> ``%s``' % \
                #      (self.fmt, int_val, conv_out_str, )

        else:

            for i in range(len(subset)):

                # Convert the last character into a tasty byte. Om nom nom.
                last_byte    = ord(subset[i])

                if self.format_type == FORMAT_DEFAULT_CHARSET:
                    # %_c 
                    if last_byte == 0:
                        conv_out_str += '\\0'
                    elif last_byte == 7:
                        conv_out_str += '\\a'
                    elif last_byte == 8:
                        conv_out_str += '\\b'
                    elif last_byte == 9:
                        conv_out_str += '\\t'
                    elif last_byte == 10:
                        conv_out_str += '\\n'
                    elif last_byte == 11:
                        conv_out_str += '\\v'
                    elif last_byte == 12:
                        conv_out_str += '\\f'
                    elif last_byte == 13:
                        conv_out_str += '\\r'
                    elif last_byte < 32 or last_byte > 127:

                        # chars are represented in 256 bits in C, thus padding
                        # goes as follows:
                        #     8 ^ 2 < 256 < 8 ^ 3
                        # Which means that the precision must be at maximum 2
                        # to properly zero-pad the value output.
                        conv_out_str[-1] = '%0.3o' % (last_byte, )
                    else:
                        conv_out_str += subset[i]

                elif self.format_type == FORMAT_ASCII_PRINT:
                    # %_u
                    if last_byte in ORD_TO_HUMANIZED:
                        conv_out_str += ORD_TO_HUMANIZED[last_byte]
                    elif last_byte > 0x7f:
                        conv_out_str += '%0.3x' % (last_byte, )
                    else:
                        conv_out_str += subset[i]

                elif self.format_type == FORMAT_NONPRINTING_CHARS:
                    # %_p
                    if last_byte < 32 or last_byte > 127:
                        conv_out_str += '.'
                    else:
                        conv_out_str += subset[i]

            conv_out_str = self.fmt % (conv_out_str, )

        return conv_out_str

    @staticmethod
    def unescape_fmt(fmt):

        """ Unescape a format string by converting some escaped control
            sequences to their unescaped forms.

            I explicitly said some, because there are some control sequences
            defined in ANSI C's printf(3) that aren't unescaped in FreeBSD's
            hexdump(1).
        """

        for escaped_val, unescaped_val in ESCAPED_CONTROL_SEQUENCES.items():
            fmt = fmt.replace(escaped_val, unescaped_val)

        return fmt

def main():

    """ main function. """

    fmt_format = '((?P<iter>\d*)\/(?P<byte>\d*)\s+)?"(?P<fmt>[^"]*)"'
    fmt_re = re.compile(fmt_format)

    parser = optparse.OptionParser(usage='%prog [file] ...')

    parser.add_option('-e', '--format-unit', dest='fmt_units',
                      action='append',
                      help='format units that define ') 

    parser.add_option('-v', '--verbose', dest='verbose', default=False,
                      action='store_true',
                      help='enable verbose printing')

    options, files = parser.parse_args()

    if options.fmt_units:
        assert len(options.fmt_units) == 1, \
            'this feature is limited to one format unit at this time'
        fmt_units = options.fmt_units
    else:
        # Default for hexsyntax.c
        fmt_units = [ '"%0.7_Ax\\n"', '"%0.7_ax " 8/1 "%0.2x " "\\n"' ]

    verbose = options.verbose

    final_offset_fmt_obj, iter_offset_fmt_obj = None, None
    fmt_objs = []

    for fmt_unit in fmt_units:

        matchiter = fmt_re.finditer(fmt_unit)

        try:

            # Iterate over the format strings and their respective
            # byte_count/iteration_count if present.

            while True:

                matchdict = matchiter.next().groupdict()

                if matchdict and 'fmt' in matchdict:

                    fmt_obj = Format(matchdict['fmt'],
                                     matchdict.get('iter', None),
                                     matchdict.get('byte', None))

                    if fmt_obj.format_type == FORMAT_OFFSET_END:
                        final_offset_fmt_obj = fmt_obj
                    elif fmt_obj.format_type == FORMAT_OFFSET_ITER:
                        iter_offset_fmt_obj = fmt_obj
                    else:
                        fmt_objs.append(fmt_obj)

                else:
                    sys.exit('Format string - %s - does not match '
                             'expected format: %s' % (fmt_unit, fmt_format))

        except StopIteration:
            pass

    max_byte_count = 0
    offset = 0
    num_bytes_consumed = 0

    for fmt_obj in fmt_objs:
        # Determine the large byte count to consume for the given format unit.
        max_byte_count = max(max_byte_count, fmt_obj.byte_count)
        #print 'bc', fmt_obj.byte_count, 'iter', fmt_obj.iteration, 'prec', \
        #    fmt_obj.precision

    for _file in files:

        fd = open(_file, 'r')

        try:

            astr_last = None
            repeat_count = 0

            # Keep on rolling until we run out of bytes to read baby!

            while not fd.closed:

                astr = ''
                iter_offset_str = None

                if iter_offset_fmt_obj:
                    iter_offset_str = iter_offset_fmt_obj.conv_str(fd,
                                                                   offset)[1]

                for fmt_obj in fmt_objs:
                    if not fd.closed:
                        num_bytes_consumed, astr_tmp = fmt_obj.conv_str(fd,
                                                                        offset)
                        astr += astr_tmp
                        offset += num_bytes_consumed

                if not verbose and astr == astr_last:
                    repeat_count += 1
                    astr = REPEAT_ASTERISK

                else:
                    repeat_count = 0
                    if not fd.closed and iter_offset_str:
                        sys.stdout.write(iter_offset_str)
                    astr_last = astr

                if repeat_count <= 1:

                    # Hold all output if the number of repeated iterations is
                    # greater than 1. This will only occur when -v isn't
                    # specified.

                    sys.stdout.write(astr)
                    sys.stdout.flush()

        except:

            if 0 < num_bytes_consumed:
                offset += num_bytes_consumed

            raise

        finally:
            if not fd.closed:
                fd.close()
            if final_offset_fmt_obj:
                sys.stdout.write(final_offset_fmt_obj.conv_str(fd, offset)[1])

if __name__ == '__main__':
    main()
